{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "# Labeling the [Lexical Relationship](https://huggingface.co/datasets/relbert/lexical_relation_classification) dataset using Autolabel\n",
    "\n",
    "This is a multi-class classification task where the input are two english words and we have to correctly classify the lexical relationship between them using one of 5 labels. "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "aacac4ae-c7f9-4dee-be3a-a2a6bfa099fa",
   "metadata": {},
   "source": [
    "## Install Autolabel\n",
    "Plus, setup your OpenAI API key, since we'll be using `gpt-3.5-turbo` as our LLM for labeling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3dc19059-2f63-44b7-9a32-8b38a11249aa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: refuel-autolabel[openai] in /usr/local/lib/python3.10/dist-packages (0.0.15)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.7.2)\n",
      "Requirement already satisfied: numpy>=1.23.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.24.1)\n",
      "Requirement already satisfied: requests>=2.27.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.28.1)\n",
      "Requirement already satisfied: datasets>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.14.5)\n",
      "Requirement already satisfied: langchain==0.0.210 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.0.210)\n",
      "Requirement already satisfied: nervaluate>=0.1.8 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.1.8)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (8.2.3)\n",
      "Requirement already satisfied: SQLAlchemy>=2.0.19 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.0.21)\n",
      "Requirement already satisfied: regex>=2023.6.3 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2023.8.8)\n",
      "Requirement already satisfied: rich>=13.3.5 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (13.5.3)\n",
      "Requirement already satisfied: scipy>=1.10.1 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.11.2)\n",
      "Requirement already satisfied: pydantic>=1.10.9 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.10.12)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.0.1+cu118)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (3.8.0)\n",
      "Requirement already satisfied: wget>=3.2 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (3.2)\n",
      "Requirement already satisfied: ipywidgets==8.0.6 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (8.0.6)\n",
      "Requirement already satisfied: jsonschema>=4.17.3 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (4.18.0)\n",
      "Requirement already satisfied: tabulate>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.9.0)\n",
      "Requirement already satisfied: typer[all]>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.9.0)\n",
      "Requirement already satisfied: simple-term-menu>=1.6.1 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.6.1)\n",
      "Requirement already satisfied: openai>=0.27.4 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.28.0)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (0.5.1)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets==8.0.6->refuel-autolabel[openai]) (6.24.0)\n",
      "Requirement already satisfied: ipython>=6.1.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets==8.0.6->refuel-autolabel[openai]) (8.14.0)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets==8.0.6->refuel-autolabel[openai]) (5.9.0)\n",
      "Requirement already satisfied: widgetsnbextension~=4.0.7 in /usr/local/lib/python3.10/dist-packages (from ipywidgets==8.0.6->refuel-autolabel[openai]) (4.0.8)\n",
      "Requirement already satisfied: jupyterlab-widgets~=3.0.7 in /usr/local/lib/python3.10/dist-packages (from ipywidgets==8.0.6->refuel-autolabel[openai]) (3.0.8)\n",
      "Requirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (6.0)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (3.8.5)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (4.0.3)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (0.5.14)\n",
      "Requirement already satisfied: langchainplus-sdk>=0.0.17 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (0.0.20)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (2.8.6)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /usr/local/lib/python3.10/dist-packages (from langchain==0.0.210->refuel-autolabel[openai]) (1.2.4)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (13.0.0)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.3.7)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.66.1)\n",
      "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.3.0)\n",
      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.70.15)\n",
      "Requirement already satisfied: fsspec[http]<2023.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2023.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.17.2)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (23.1)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=4.17.3->refuel-autolabel[openai]) (23.1.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=4.17.3->refuel-autolabel[openai]) (2023.6.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=4.17.3->refuel-autolabel[openai]) (0.29.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=4.17.3->refuel-autolabel[openai]) (0.8.10)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.5)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2023.3)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.12.7)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.3.5->refuel-autolabel[openai]) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.15.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.2.0)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=2.0.19->refuel-autolabel[openai]) (2.0.2)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.12.4)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (1.11.1)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.0)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.1.2)\n",
      "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (2.0.0)\n",
      "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->refuel-autolabel[openai]) (3.25.0)\n",
      "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->refuel-autolabel[openai]) (15.0.7)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer[all]>=0.9.0->refuel-autolabel[openai]) (8.1.7)\n",
      "Requirement already satisfied: colorama<0.5.0,>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from typer[all]>=0.9.0->refuel-autolabel[openai]) (0.4.4)\n",
      "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]>=0.9.0->refuel-autolabel[openai]) (1.5.3)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.210->refuel-autolabel[openai]) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.210->refuel-autolabel[openai]) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.210->refuel-autolabel[openai]) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.210->refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.210->refuel-autolabel[openai]) (3.20.1)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.210->refuel-autolabel[openai]) (0.9.0)\n",
      "Requirement already satisfied: comm>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.1.3)\n",
      "Requirement already satisfied: debugpy>=1.6.5 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (1.6.7)\n",
      "Requirement already satisfied: jupyter-client>=6.1.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (8.3.0)\n",
      "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (5.3.1)\n",
      "Requirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.1.6)\n",
      "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (1.5.6)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (5.9.5)\n",
      "Requirement already satisfied: pyzmq>=20 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (25.1.0)\n",
      "Requirement already satisfied: tornado>=6.1 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (6.3.2)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.2.0)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.18.2)\n",
      "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (3.0.39)\n",
      "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.6.2)\n",
      "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (4.8.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->refuel-autolabel[openai]) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->refuel-autolabel[openai]) (1.2.1)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.8.3)\n",
      "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel>=4.5.1->ipywidgets==8.0.6->refuel-autolabel[openai]) (3.10.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.2.6)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.210->refuel-autolabel[openai]) (1.0.0)\n",
      "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (2.2.1)\n",
      "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/dist-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.0.6->refuel-autolabel[openai]) (0.2.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbdeca2f-dd20-4634-b3f8-1b3ed45c4705",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-\"\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8eb09918-494a-42ef-86a7-df93b3ce4284",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Download the dataset\n",
    "\n",
    "This dataset is available to install via Autolabel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b4c116ce-3294-45c2-9158-eac929f03a95",
   "metadata": {},
   "outputs": [],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data(\"lexical_relation\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5f038fcc-9ec0-4f2d-b84d-2e963656c6bd",
   "metadata": {},
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "33d47b67-0718-4289-bb59-989d851d09ed",
   "metadata": {},
   "source": [
    "### First labeling run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open(\"config_lexical_relation.json\") as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1fad4e85-d598-413d-9b01-9690653a05ad",
   "metadata": {},
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `classification` (since it's a classification task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are an expert at understanding bank customers support complaints and queries...` (how we describe the task to the LLM)\n",
    "* `prompt.labels`: `['age_limit', 'apple_pay_or_google_pay', 'atm_support', ...]` (the full list of labels to choose from)\n",
    "* `prompt.few_shot_num`: 10 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6a3610fd-721e-44de-9b2c-2cd73ec86bba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'LexicalRelationClassification',\n",
       " 'task_type': 'classification',\n",
       " 'dataset': {'label_column': 'label', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are an expert at understanding lexical relationships.\\n Given two words A and B, classify the relationship between them into one of: {labels}. RANDOM refers to an arbitrary relationship. SYN means that the words are synonyms. ANT means that the two words are antonyms. HYPER means that the two words are hypernyms. PART_OF means that the second word is a subset of the first word.',\n",
       "  'output_guidelines': 'You will answer with just the the correct output label and nothing else.',\n",
       "  'labels': ['RANDOM', 'SYN', 'ANT', 'HYPER', 'PART_OF'],\n",
       "  'few_shot_examples': 'seed.csv',\n",
       "  'few_shot_selection': 'semantic_similarity',\n",
       "  'few_shot_num': 10,\n",
       "  'example_template': 'Input: {example}\\nOutput: {label}'}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f867f539cd24a83bc8020f736d5819d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $4.8362 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0024 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$4.8362\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0024\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">You are an expert at understanding lexical relationships.\n",
       " Given two words A and B, classify the relationship between them into one of: RANDOM\n",
       "SYN\n",
       "ANT\n",
       "HYPER\n",
       "PART_OF. RANDOM refers to an arbitrary relationship. SYN means that the words are synonyms. ANT means that the two \n",
       "words are antonyms. HYPER means that the two words are hypernyms. PART_OF means that the second word is a subset of\n",
       "the first word.\n",
       "\n",
       "You will answer with just the the correct output label and nothing else.\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Input: A: bowl\n",
       "B: dish\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: ignorance\n",
       "B: dish\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: chicken\n",
       "B: food\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: america\n",
       "B: country\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: apartment\n",
       "B: play\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: beef\n",
       "B: team\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: captain\n",
       "B: team\n",
       "Output: PART_OF\n",
       "\n",
       "Input: A: actress\n",
       "B: person\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: monitor\n",
       "B: television\n",
       "Output: PART_OF\n",
       "\n",
       "Input: A: hit\n",
       "B: food\n",
       "Output: RANDOM\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Input: A: company\n",
       "B: dish\n",
       "Output: \n",
       "</pre>\n"
      ],
      "text/plain": [
       "You are an expert at understanding lexical relationships.\n",
       " Given two words A and B, classify the relationship between them into one of: RANDOM\n",
       "SYN\n",
       "ANT\n",
       "HYPER\n",
       "PART_OF. RANDOM refers to an arbitrary relationship. SYN means that the words are synonyms. ANT means that the two \n",
       "words are antonyms. HYPER means that the two words are hypernyms. PART_OF means that the second word is a subset of\n",
       "the first word.\n",
       "\n",
       "You will answer with just the the correct output label and nothing else.\n",
       "\n",
       "Some examples with their output answers are provided below:\n",
       "\n",
       "Input: A: bowl\n",
       "B: dish\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: ignorance\n",
       "B: dish\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: chicken\n",
       "B: food\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: america\n",
       "B: country\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: apartment\n",
       "B: play\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: beef\n",
       "B: team\n",
       "Output: RANDOM\n",
       "\n",
       "Input: A: captain\n",
       "B: team\n",
       "Output: PART_OF\n",
       "\n",
       "Input: A: actress\n",
       "B: person\n",
       "Output: HYPER\n",
       "\n",
       "Input: A: monitor\n",
       "B: television\n",
       "Output: PART_OF\n",
       "\n",
       "Input: A: hit\n",
       "B: food\n",
       "Output: RANDOM\n",
       "\n",
       "Now I want you to label the following example:\n",
       "Input: A: company\n",
       "B: dish\n",
       "Output: \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dry-run -- this tells us how much this will cost and shows an example prompt\n",
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/lexical_relation/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-29 00:03:26 autolabel.labeler INFO: Task run already exists.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">There is an existing task with following details: <span style=\"color: #808000; text-decoration-color: #808000\">id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'1029336087'</span> <span style=\"color: #808000; text-decoration-color: #808000\">created_at</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">datetime</span><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">.datetime</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2023</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">29</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, \n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">415913</span><span style=\"font-weight: bold\">)</span> <span style=\"color: #808000; text-decoration-color: #808000\">task_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'7071645cb8e8bf65daaf1770b1b51d08'</span> <span style=\"color: #808000; text-decoration-color: #808000\">dataset_id</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'a520beb48d484eec1cfb9d39d0152ece'</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">current_index</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> <span style=\"color: #808000; text-decoration-color: #808000\">output_file</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'LexicalRelationClassification_labeled.csv'</span> <span style=\"color: #808000; text-decoration-color: #808000\">status</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">TaskStatus.ACTIVE:</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008000; text-decoration-color: #008000\">'active'</span><span style=\"font-weight: bold\">&gt;</span> \n",
       "<span style=\"color: #808000; text-decoration-color: #808000\">error</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span> <span style=\"color: #808000; text-decoration-color: #808000\">metrics</span>=<span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "There is an existing task with following details: \u001b[33mid\u001b[0m=\u001b[32m'1029336087'\u001b[0m \u001b[33mcreated_at\u001b[0m=\u001b[1;35mdatetime\u001b[0m\u001b[1;35m.datetime\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m2023\u001b[0m, \u001b[1;36m9\u001b[0m, \u001b[1;36m29\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m2\u001b[0m, \n",
       "\u001b[1;36m24\u001b[0m, \u001b[1;36m415913\u001b[0m\u001b[1m)\u001b[0m \u001b[33mtask_id\u001b[0m=\u001b[32m'7071645cb8e8bf65daaf1770b1b51d08'\u001b[0m \u001b[33mdataset_id\u001b[0m=\u001b[32m'a520beb48d484eec1cfb9d39d0152ece'\u001b[0m \n",
       "\u001b[33mcurrent_index\u001b[0m=\u001b[1;36m67\u001b[0m \u001b[33moutput_file\u001b[0m=\u001b[32m'LexicalRelationClassification_labeled.csv'\u001b[0m \u001b[33mstatus\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mTaskStatus.ACTIVE:\u001b[0m\u001b[39m \u001b[0m\u001b[32m'active'\u001b[0m\u001b[1m>\u001b[0m \n",
       "\u001b[33merror\u001b[0m=\u001b[3;35mNone\u001b[0m \u001b[33mmetrics\u001b[0m=\u001b[3;35mNone\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluating the existing task<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluating the existing task\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 66      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0             </span>│\n",
       "└──────────┴─────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m66     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└──────────┴─────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples labeled so far.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m67\u001b[0m examples labeled so far.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Do you want to resume the task? <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">[y/n]</span>: </pre>\n"
      ],
      "text/plain": [
       "Do you want to resume the task? \u001b[1;35m[y/n]\u001b[0m: "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " n\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Deleted the existing task and starting a new one<span style=\"color: #808000; text-decoration-color: #808000\">...</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Deleted the existing task and starting a new one\u001b[33m...\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5495eb65f86c4645bb45da2a060a45da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">classification_report:\n",
       "              precision    recall  f1-score   support\n",
       "\n",
       "         ANT       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.78</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.78</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.78</span>         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>\n",
       "       HYPER       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.67</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.29</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.40</span>         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span>\n",
       "     PART_OF       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.18</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.67</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.29</span>         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>\n",
       "      RANDOM       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.96</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.89</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.92</span>        <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">75</span>\n",
       "         SYN       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.71</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.83</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.77</span>         <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>\n",
       "\n",
       "    accuracy                           <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.83</span>       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>\n",
       "   macro avg       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.66</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.69</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.63</span>       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>\n",
       "weighted avg       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.88</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.83</span>      <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.85</span>       <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "classification_report:\n",
       "              precision    recall  f1-score   support\n",
       "\n",
       "         ANT       \u001b[1;36m0.78\u001b[0m      \u001b[1;36m0.78\u001b[0m      \u001b[1;36m0.78\u001b[0m         \u001b[1;36m9\u001b[0m\n",
       "       HYPER       \u001b[1;36m0.67\u001b[0m      \u001b[1;36m0.29\u001b[0m      \u001b[1;36m0.40\u001b[0m         \u001b[1;36m7\u001b[0m\n",
       "     PART_OF       \u001b[1;36m0.18\u001b[0m      \u001b[1;36m0.67\u001b[0m      \u001b[1;36m0.29\u001b[0m         \u001b[1;36m3\u001b[0m\n",
       "      RANDOM       \u001b[1;36m0.96\u001b[0m      \u001b[1;36m0.89\u001b[0m      \u001b[1;36m0.92\u001b[0m        \u001b[1;36m75\u001b[0m\n",
       "         SYN       \u001b[1;36m0.71\u001b[0m      \u001b[1;36m0.83\u001b[0m      \u001b[1;36m0.77\u001b[0m         \u001b[1;36m6\u001b[0m\n",
       "\n",
       "    accuracy                           \u001b[1;36m0.83\u001b[0m       \u001b[1;36m100\u001b[0m\n",
       "   macro avg       \u001b[1;36m0.66\u001b[0m      \u001b[1;36m0.69\u001b[0m      \u001b[1;36m0.63\u001b[0m       \u001b[1;36m100\u001b[0m\n",
       "weighted avg       \u001b[1;36m0.88\u001b[0m      \u001b[1;36m0.83\u001b[0m      \u001b[1;36m0.85\u001b[0m       \u001b[1;36m100\u001b[0m\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Actual Cost: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0421</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Actual Cost: \u001b[1;36m0.0421\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.83     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└──────────┴─────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.83    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└──────────┴─────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now, do the actual labeling\n",
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a0a4bc7-b62e-4ad3-96a5-9e04fe53c24b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
